{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "sys.path.append('../utils/')\n",
    "sys.path.append('../feature/')\n",
    "\n",
    "\n",
    "from help import score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(102477, 12)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_hdf('../data/cache/train_cut_char.hdf')\n",
    "data['q1_cut'] = data['q1'].map(lambda x: jieba.cut(x.strip()))\n",
    "data['q2_cut'] = data['q2'].map(lambda x:jieba.cut(x.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['q1_len'] = data.q1.map(lambda x:len(x))\n",
    "data['q2_len'] = data.q2.map(lambda x:len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>102477.000000</td>\n",
       "      <td>102477.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>13.390956</td>\n",
       "      <td>13.404520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.111260</td>\n",
       "      <td>6.134927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>97.000000</td>\n",
       "      <td>112.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              q1_len         q2_len\n",
       "count  102477.000000  102477.000000\n",
       "mean       13.390956      13.404520\n",
       "std         6.111260       6.134927\n",
       "min         5.000000       3.000000\n",
       "25%        10.000000      10.000000\n",
       "50%        12.000000      12.000000\n",
       "75%        15.000000      15.000000\n",
       "max        97.000000     112.000000"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[['q1_len','q2_len']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def dif(row):\n",
    "    q1 = set(row[u'q1_cut'])\n",
    "    q2 = set(row[u'q2_cut'])\n",
    "    return q1.difference(q2)\n",
    "def dif2(row):\n",
    "    q1 = set(row[u'q1_cut'])\n",
    "    q2 = set(row[u'q2_cut'])\n",
    "    return q2.difference(q1)\n",
    "\n",
    "\n",
    "data['char_df1']=data.apply(lambda x:dif(x),axis=1)\n",
    "data['char_df2']=data.apply(lambda x:dif2(x),axis=1)\n",
    "# def zero12(row):\n",
    "#     return len(row['char_df1'])==0 or len(row['char_df2'])==0)\n",
    "# def zero21(row):\n",
    "#     return int(len(row['char_df1'])==0 or len(row['char_df2'])==0)\n",
    "data['zero1'] = data['char_df1'].map(lambda x:len(x))\n",
    "data['zero2'] = data['char_df2'].map(lambda x:len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['len_x1'] = data['q1_cut'].map(lambda x:len(list(x)))\n",
    "data['len_x2'] = data['q2_cut'].map(lambda x:len(list(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_str(x):\n",
    "    #x = re.sub('***',u'X',x)\n",
    "\n",
    "    punc = \"蚂蚁 为何 为什么 怎么 如何 了 吗  的 ！？。 ＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.\"\n",
    "    punc = punc.decode(\"utf-8\")\n",
    "    return re.sub(ur\"[%s]+\" %punc, \"\", x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                   ﻿ 怎么 更改 花 呗 手机号码\n",
       "1                         也 开 不了 花 呗 ， 就 这样 了 ？ 完事 了\n",
       "2                                 花 呗 冻结 以后 还 能 开通 吗\n",
       "3                                       如何 得知 关闭 借 呗\n",
       "4                                          花 呗 扫码 付钱\n",
       "5                                   花 呗 逾期 后 不能 分期 吗\n",
       "6                                          花 呗 分期 清空\n",
       "7                                       借 呗 逾期 短信 通知\n",
       "8                        借 呗 即将 到期 要 还 的 账单 还 能 分期 吗\n",
       "9                                花 呗 为什么 不能 支付 手机 交易\n",
       "10                       在 吗 ， 双 * * * 有 临时 花 呗 额度 吗\n",
       "11                                  我 什么 时候 开通 了 花 呗\n",
       "12             花 呗 每月 还 了 最低 还款 后 还 会 有 万分之 五 的 利息 吗\n",
       "13                              我 想 用 蚂蚁 借 呗 怎么 用 不了\n",
       "14                                      花 呗 也 不能 用 了\n",
       "15                                 花 呗 的 安全 没有 验证 成功\n",
       "16                                       我 开通 不了 借 呗\n",
       "17                                借 呗 还款 了 ， 额度 未 恢复\n",
       "18                     就是 我花 呗 忘记 还款 了 。 逾期 一天 。 有事吗\n",
       "19                              我 问 你 我 借 呗 怎么 要 还 钱\n",
       "20                                是否 有花 呗 就 不 可以 借 呗\n",
       "21                                    闲鱼 可以 使用 花 呗 吗\n",
       "22                                      花 呗 可以 买手机 吗\n",
       "23                                      借 呗 每月 还款 时间\n",
       "24                                     花 呗 更改 绑定 银行卡\n",
       "25                                      人脸 验证 开通 花 呗\n",
       "26                                 借 呗 可以 提前 还 第一期 吗\n",
       "27                          花 呗 付款 成功 为何 美团 显示 支付 超时\n",
       "28                         花 呗 我 已经 还 了 ， 现在 要 还 借 呗\n",
       "29                                 国外 账户 可以 开通 借 呗 吗\n",
       "                           ...                      \n",
       "70                             我花 呗 额度 能 不能 给 我 调 高点\n",
       "71                  花 呗 之前 绑定 的 手机号码 现在 没用 了 ， 怎么 解绑\n",
       "72                                    花 呗 线下 还款 是 什么\n",
       "73    我 * * * 月份 花 呗 消费 是 * * * 元 ， 怎么 要 我 还 * * * 元\n",
       "74                                      用花 呗 刷卡 可以 吗\n",
       "75                                     花 呗 分期 卖家 手续费\n",
       "76                                    花 呗 自动 余额 宝 还款\n",
       "77                                  我 的   借 呗   在 哪儿\n",
       "78                           我 的 花 呗 没有 绑定 银行卡 怎么 还款\n",
       "79                             为什么 双 十二 没有 花 呗 零时 额度\n",
       "80                             借 呗 有 晚 一天 还有 没有 逾期 呀\n",
       "81                                 双 * * * 零时 花 呗 额度\n",
       "82                                付款 方式 添加 花 呗 怎么 添加\n",
       "83                             使用 花 呗 支付 会 有 短信 提醒 吗\n",
       "84                                        借 呗 能 提款 不\n",
       "85                  我 的 花 呗 怎么 不能 用 这个 月 还 没有 给 我 额度\n",
       "86                                付款 金额 多少 可以 使用 花 呗\n",
       "87                                      怎么 花 呗 不能 支付\n",
       "88                            花 呗 到 还款 日期 还 不了 会 怎么样\n",
       "89                          为什么 我 的 收钱 码 打 不 开花 呗 收钱\n",
       "90                                我 满足 开通 借 呗 的 条件 吗\n",
       "91                        怎么 感觉 蚂蚁 花 呗 钱 越 还 欠 的 越 多\n",
       "92                                    借 呗 可以 一次 结清 吗\n",
       "93                办理 花 呗 分期 后 可以 提前 还清 账单 吗 ？ 利息 高 吗\n",
       "94                                      花 呗 有 退 运费 的\n",
       "95               我 前天   用 了 花 呗 ， 今天 能够 还款 不 ， 怎么 还款\n",
       "96                                          找 不到 花 呗\n",
       "97                              我 绑定 的 银行卡 还 不 上 借 呗\n",
       "98                                    我 的 花 呗 收 不了 款\n",
       "99                              花 呗 支付 可以 使用 购物 津贴 吗\n",
       "Name: q1, Length: 100, dtype: object"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(100).q1.map(lambda x: ' '.join(jieba.cut(x.strip())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                ﻿ 更改 花 呗 手机号码\n",
       "1                              也 开不花 呗 就 这样 完事\n",
       "2                             花 呗 冻结 以后 还 能 开通\n",
       "3                                    得知 关闭 借 呗\n",
       "4                                    花 呗 扫码 付钱\n",
       "5                               花 呗 逾期 后 不能 分期\n",
       "6                                    花 呗 分期 清空\n",
       "7                                 借 呗 逾期 短信 通知\n",
       "8                      借 呗 即将 到期 要 还 账单 还 能 分期\n",
       "9                              花 呗 不能 支付 手机 交易\n",
       "10                       在 双 * * * 有 临时 花 呗 额度\n",
       "11                                 我 时候 开通 花 呗\n",
       "12             花 呗 每月 还 最低 还款 后 还 会 有 万分之 五 利息\n",
       "13                               我 想 用 借 呗 用 不\n",
       "14                                  花 呗 也 不能 用\n",
       "15                             花 呗 安全 没有 验证 成功\n",
       "16                                   我 开通 不借 呗\n",
       "17                              借 呗 还款额 度 未 恢复\n",
       "18                      就是 我花 呗 忘记 还款 逾期 一天 有事\n",
       "19                           我 问 你 我 借 呗 要 还 钱\n",
       "20                          是否 有花 呗 就 不 可以 借 呗\n",
       "21                                闲鱼 可以 使用 花 呗\n",
       "22                                  花 呗 可以 买手机\n",
       "23                                借 呗 每月 还款 时间\n",
       "24                               花 呗 更改 绑定 银行卡\n",
       "25                                人脸 验证 开通 花 呗\n",
       "26                             借 呗 可以 提前 还 第一期\n",
       "27                       花 呗 付款 成功 美团 显示 支付 超时\n",
       "28                       花 呗 我 已经 还 现在 要 还 借 呗\n",
       "29                             国外 账户 可以 开通 借 呗\n",
       "                        ...                   \n",
       "70                       我花 呗 额度 能 不能 给 我 调 高点\n",
       "71                     花 呗 之前 绑定 手机号码 现在 没用 解绑\n",
       "72                                 花 呗 线下 还款 是\n",
       "73    我 * * * 月份 花 呗 消费 是 * * * 元要 我 还 * * * 元\n",
       "74                                  用花 呗 刷卡 可以\n",
       "75                               花 呗 分期 卖家 手续费\n",
       "76                              花 呗 自动 余额 宝 还款\n",
       "77                                  我 借 呗 在 哪儿\n",
       "78                           我花 呗 没有 绑定 银行卡 还款\n",
       "79                           双 十二 没有 花 呗 零时 额度\n",
       "80                       借 呗 有 晚 一天 还有 没有 逾期 呀\n",
       "81                           双 * * * 零时 花 呗 额度\n",
       "82                             付款 方式 添加 花 呗 添加\n",
       "83                         使用 花 呗 支付 会 有 短信 提醒\n",
       "84                                  借 呗 能 提款 不\n",
       "85                  我花 呗 不能 用 这个 月 还 没有 给 我 额度\n",
       "86                          付款 金额 多少 可以 使用 花 呗\n",
       "87                                   花 呗 不能 支付\n",
       "88                          花 呗 到 还款 日期 还 不会 样\n",
       "89                          我 收钱 码 打 不 开花 呗 收钱\n",
       "90                              我 满足 开通 借 呗 条件\n",
       "91                          感觉 花 呗 钱 越 还 欠 越 多\n",
       "92                                借 呗 可以 一次 结清\n",
       "93                办理 花 呗 分期 后 可以 提前 还清 账单 利息 高\n",
       "94                                  花 呗 有 退 运费\n",
       "95                     我 前天 用花 呗 今天 能够 还款 不 还款\n",
       "96                                    找 不到 花 呗\n",
       "97                          我 绑定 银行卡 还 不 上 借 呗\n",
       "98                                    我花 呗 收不款\n",
       "99                          花 呗 支付 可以 使用 购物 津贴\n",
       "Name: q1, Length: 100, dtype: object"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(100).q1.apply(lambda x:clean_str(x)).map(lambda x: ' '.join(jieba.cut(x.strip())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      \n",
       "1      \n",
       "2      \n",
       "3      \n",
       "4      \n",
       "5      \n",
       "6      \n",
       "7      \n",
       "8      \n",
       "9      \n",
       "10     \n",
       "11     \n",
       "12     \n",
       "13     \n",
       "14     \n",
       "15     \n",
       "16     \n",
       "17     \n",
       "18     \n",
       "19     \n",
       "20     \n",
       "21     \n",
       "22     \n",
       "23     \n",
       "24     \n",
       "25     \n",
       "26     \n",
       "27     \n",
       "28     \n",
       "29     \n",
       "     ..\n",
       "70     \n",
       "71     \n",
       "72     \n",
       "73     \n",
       "74     \n",
       "75     \n",
       "76     \n",
       "77     \n",
       "78     \n",
       "79     \n",
       "80     \n",
       "81     \n",
       "82     \n",
       "83     \n",
       "84     \n",
       "85     \n",
       "86     \n",
       "87     \n",
       "88     \n",
       "89     \n",
       "90     \n",
       "91     \n",
       "92     \n",
       "93     \n",
       "94     \n",
       "95     \n",
       "96     \n",
       "97     \n",
       "98     \n",
       "99     \n",
       "Name: q1_cut, Length: 100, dtype: object"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(100).q1_cut.map(lambda x:' '.join(list(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    83792\n",
       "1    18685\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80000, 4)\n",
      "train first model....\n",
      "[0]\ttrain-logloss:0.59163\tvalid-logloss:0.582237\n",
      "Multiple eval metrics have been passed: 'valid-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-logloss hasn't improved in 150 rounds.\n",
      "[10]\ttrain-logloss:0.476803\tvalid-logloss:0.441611\n",
      "[20]\ttrain-logloss:0.476621\tvalid-logloss:0.440405\n",
      "[30]\ttrain-logloss:0.476619\tvalid-logloss:0.440381\n",
      "[40]\ttrain-logloss:0.476618\tvalid-logloss:0.44038\n",
      "[50]\ttrain-logloss:0.476618\tvalid-logloss:0.440381\n",
      "[60]\ttrain-logloss:0.476617\tvalid-logloss:0.440381\n",
      "[70]\ttrain-logloss:0.476617\tvalid-logloss:0.440381\n",
      "[80]\ttrain-logloss:0.476617\tvalid-logloss:0.440381\n",
      "[90]\ttrain-logloss:0.476617\tvalid-logloss:0.440381\n",
      "[100]\ttrain-logloss:0.476616\tvalid-logloss:0.440381\n",
      "[110]\ttrain-logloss:0.476616\tvalid-logloss:0.440381\n",
      "[120]\ttrain-logloss:0.476616\tvalid-logloss:0.440381\n",
      "[130]\ttrain-logloss:0.476616\tvalid-logloss:0.440381\n",
      "[140]\ttrain-logloss:0.476616\tvalid-logloss:0.44038\n",
      "[150]\ttrain-logloss:0.476616\tvalid-logloss:0.44038\n",
      "[160]\ttrain-logloss:0.476616\tvalid-logloss:0.44038\n",
      "[170]\ttrain-logloss:0.476616\tvalid-logloss:0.44038\n",
      "[180]\ttrain-logloss:0.476616\tvalid-logloss:0.44038\n",
      "Stopping. Best iteration:\n",
      "[33]\ttrain-logloss:0.476619\tvalid-logloss:0.44038\n",
      "\n",
      "0\n",
      "3642\n",
      "('p r f1 ', 0.0, 0.0, 0.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import gc\n",
    "import xgboost as xgb\n",
    "from math import floor\n",
    "from time import time\n",
    "\n",
    "\n",
    "feats = ['zero1','zero2','len_x1','len_x2']\n",
    "\n",
    "\n",
    "# Train model\n",
    "X_train = data[:80000][feats]\n",
    "print(X_train.shape)\n",
    "Y_train = data[:80000]['label']\n",
    "\n",
    "X_valid = data[80000:][feats]\n",
    "Y_valid =data[80000:]['label']\n",
    "\n",
    "params = {}\n",
    "params['objective'] = 'binary:logistic'\n",
    "params['eval_metric'] = 'logloss'\n",
    "params[\"max_depth\"] = 8\n",
    "params[\"silent\"] = 1\n",
    "params[\"seed\"] = 1632\n",
    "\n",
    "d_train = xgb.DMatrix(X_train, label=Y_train)\n",
    "d_valid = xgb.DMatrix(X_valid, label=Y_valid)\n",
    "watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "\n",
    "print('train first model....')\n",
    "bst = xgb.train(params, d_train, 50000, watchlist,early_stopping_rounds=150, verbose_eval=10)\n",
    "\n",
    "\n",
    "# This is used to determine the best round to stop on. This is not the ideal traning method,\n",
    "# but it does allow training the model on the entire data set, which gave some gains over\n",
    "# using a only a train/test method + early stopping.\n",
    "n_iter = bst.best_iteration\n",
    "\n",
    "pred = bst.predict(d_valid)\n",
    "pred = [int(x>0.5) for x in pred]\n",
    "print(sum(pred))\n",
    "print(sum(Y_valid))\n",
    "pre, rec, f1 = score(Y_valid, pred)\n",
    "print('p r f1 ', pre, rec, f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py2]",
   "language": "python",
   "name": "conda-env-py2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
